#include"ORCL.h"

struct ibv_device* sync_resolve_port_index(struct orcl_context_sync* sync_ctx, int port_index){
	struct ibv_device **dev_list;
	int num_devices = 0;

	assert(port_index >= 0);

	// 真正意义上的设备编号号和设备的端口编号号
	sync_ctx->device_id = -1;
	sync_ctx->dev_port_id = -1;

	dev_list = ibv_get_device_list(&num_devices);
	CPE(!dev_list, "ORCL: Failed to get IB devices list", 0);

	// 这个是声明要使用第几个端口
	int ports_to_discover = port_index;
	int dev_i;

	// 遍历所有的设备，并且遍历这些设备的活跃端口，
	for (dev_i = 0; dev_i < num_devices; dev_i++)
	{
		struct ibv_context *ctx = ibv_open_device(dev_list[dev_i]);
		CPE(!ctx, "ORCL: Couldn't open device", 0);

		struct ibv_device_attr device_attr;
		memset(&device_attr, 0, sizeof(device_attr));
		if (ibv_query_device(ctx, &device_attr))
		{
			printf("ORCL: Could not query device: %d\n", dev_i);
			exit(-1);
		}

		// 硬件上的编号从1开始，所以从1开始查
		uint8_t port_i;
		for (port_i = 1; port_i <= device_attr.phys_port_cnt; port_i++)
		{

			/* Count this port only if it is enabled */
			struct ibv_port_attr port_attr;

			// 遍历所有的port
			if (ibv_query_port(ctx, port_i, &port_attr) != 0)
			{
				printf("ORCL: Could not query port %d of device %d\n",
					   port_i, dev_i);
				exit(-1);
			}

			// 跳过没有激活的端口
			if (port_attr.phys_state != IBV_PORT_ACTIVE &&
				port_attr.phys_state != IBV_PORT_ACTIVE_DEFER)
			{
				printf("ORCL: Ignoring port %d of device %d. State is %s\n",
					   port_i, dev_i, ibv_port_state_str((ibv_port_state)port_attr.phys_state));

				continue;
			}

			// 这里代表了已经找到对应的端口了，不仅要打印对应端口的逻辑id，也要打出lid
			if (ports_to_discover == 0)
			{
				printf("ORCL: port index %d resolved to device %d, port %d\n",
					   port_index, dev_i, port_i);

				/* Fill the device ID and device-local port ID */
				sync_ctx->device_id = dev_i;
				sync_ctx->dev_port_id = port_i;

				if (ibv_close_device(ctx))
				{
					fprintf(stderr, "ORCL: Couldn't release context\n");
					assert(false);
				}

				return dev_list[sync_ctx->device_id];
			}

			ports_to_discover--;
		}

		if (ibv_close_device(ctx))
		{
			fprintf(stderr, "ORCL: Couldn't release context\n");
			assert(false);
		}
	}

	/* If we come here, port resolution failed */
	assert(sync_ctx->device_id == -1 && sync_ctx->dev_port_id == -1);
	fprintf(stderr, "ORCL: Invalid port index %d. Exiting.\n", port_index);
	exit(-1);
}

struct ibv_device *orcl_resolve_port_index(struct orcl_context *orcl_ctx, int port_index)
{
	struct ibv_device **dev_list;
	int num_devices = 0;

	assert(port_index >= 0);

	// 真正意义上的设备编号号和设备的端口编号号
	orcl_ctx->device_id = -1;
	orcl_ctx->dev_port_id = -1;

	dev_list = ibv_get_device_list(&num_devices);
	CPE(!dev_list, "ORCL: Failed to get IB devices list", 0);

	// 这个是声明要使用第几个端口
	int ports_to_discover = port_index;
	int dev_i;

	// 遍历所有的设备，并且遍历这些设备的活跃端口，
	for (dev_i = 0; dev_i < num_devices; dev_i++)
	{
		struct ibv_context *ctx = ibv_open_device(dev_list[dev_i]);
		CPE(!ctx, "ORCL: Couldn't open device", 0);

		struct ibv_device_attr device_attr;
		memset(&device_attr, 0, sizeof(device_attr));
		if (ibv_query_device(ctx, &device_attr))
		{
			printf("ORCL: Could not query device: %d\n", dev_i);
			exit(-1);
		}

		// 硬件上的编号从1开始，所以从1开始查
		uint8_t port_i;
		for (port_i = 1; port_i <= device_attr.phys_port_cnt; port_i++)
		{

			/* Count this port only if it is enabled */
			struct ibv_port_attr port_attr;

			// 遍历所有的port
			if (ibv_query_port(ctx, port_i, &port_attr) != 0)
			{
				printf("ORCL: Could not query port %d of device %d\n",
					   port_i, dev_i);
				exit(-1);
			}

			// 跳过没有激活的端口
			if (port_attr.phys_state != IBV_PORT_ACTIVE &&
				port_attr.phys_state != IBV_PORT_ACTIVE_DEFER)
			{
				printf("ORCL: Ignoring port %d of device %d. State is %s\n",
					   port_i, dev_i, ibv_port_state_str((ibv_port_state)port_attr.phys_state));

				continue;
			}

			// 这里代表了已经找到对应的端口了，不仅要打印对应端口的逻辑id，也要打出lid
			if (ports_to_discover == 0)
			{
				printf("ORCL: port index %d resolved to device %d, port %d\n",
					   port_index, dev_i, port_i);

				/* Fill the device ID and device-local port ID */
				orcl_ctx->device_id = dev_i;
				orcl_ctx->dev_port_id = port_i;

				if (ibv_close_device(ctx))
				{
					fprintf(stderr, "ORCL: Couldn't release context\n");
					assert(false);
				}

				return dev_list[orcl_ctx->device_id];
			}

			ports_to_discover--;
		}

		if (ibv_close_device(ctx))
		{
			fprintf(stderr, "ORCL: Couldn't release context\n");
			assert(false);
		}
	}

	/* If we come here, port resolution failed */
	assert(orcl_ctx->device_id == -1 && orcl_ctx->dev_port_id == -1);
	fprintf(stderr, "ORCL: Invalid port index %d. Exiting.\n", port_index);
	exit(-1);
}

void* orcl_malloc_socket(int shm_key, int size, int socket_id){

#if USE_HUGE_PAGE == 1

    int shmid = shmget(shm_key,
		size, IPC_CREAT | IPC_EXCL | 0666 | SHM_HUGETLB);

    if(shmid == -1) {
		switch(errno) {
			case EACCES:
				orcl_red_printf("ORCL: SHM malloc error: Insufficient permissions."
					" (SHM key = %d)\n", shm_key);
				break;
			case EEXIST:
				orcl_red_printf("ORCL: SHM malloc error: Already exists."
					" (SHM key = %d)\n", shm_key);
				break;
			case EINVAL:
				orcl_red_printf("ORCL: SHM malloc error: SHMMAX/SHMIN mismatch."
					" (SHM key = %d, size = %d)\n", shm_key, size);
				break;
			case ENOMEM:
				orcl_red_printf("ORCL: SHM malloc error: Insufficient memory."
					" (SHM key = %d, size = %d)\n", shm_key, size);
				break;
			default:
				orcl_red_printf("ORCL: SHM malloc error: A wild SHM error: %s.\n",
					strerror(errno));
				break;
		}
		assert(false);
	}

    // 申请真正的大页缓冲区
    void *buf = shmat(shmid, NULL, 0);
	if(buf == NULL) {
		printf("ORCL: SHM malloc error: shmat() failed for key %d\n", shm_key);
		exit(-1);
	}

    // 将共享内存对应的处理器绑定
    /* Bind the buffer to this socket */
	const unsigned long nodemask = (1 << socket_id);
	int ret = mbind(buf, size, MPOL_BIND, &nodemask, 32, 0);
	if(ret != 0) {
		printf("ORCL: SHM malloc error. mbind() failed for key %d\n", shm_key);
		exit(-1);
	}

	return buf;

#endif

#if USE_HUGE_PAGE == 0

	return orcl_malloc_socket_no_hugepage(size, socket_id);

#endif

}

// 不申请大页
void* orcl_malloc_socket_no_hugepage(int size, int socket_id){
	void* buf = NULL;
	buf = (void*)memalign(65536, size);

	if(buf == NULL) {
		printf("ORCL: fail to alloc memory\n");
		exit(-1);
	}

	/* Bind the buffer to this socket */
	const unsigned long nodemask = (1 << socket_id);
	int ret = mbind(buf, size, MPOL_BIND, &nodemask, 32, 0);
	if(ret != 0) {
		printf("ORCL: fail to mbind memory\n");
		exit(-1);
	}

	return buf;
}


uint16_t orcl_get_local_lid(struct ibv_context *ctx, int dev_port_id){
	assert(ctx != NULL);

	assert(dev_port_id >= 1);

	struct ibv_port_attr attr;
	if(ibv_query_port(ctx, dev_port_id, &attr)) {
		printf("ORCL: ibv_query_port on port %d of device %s failed! Exiting.\n",
			dev_port_id, ibv_get_device_name(ctx->device));
		assert(false);
	}

	return attr.lid;
}

// 在同步接口中拷贝数据到设备
void sync_cpy_to_device(struct orcl_context_sync* sync_ctx, uint8_t* source_buf, int source_offset, int dm_index, int dest_offset, int size){
	assert(sync_ctx != NULL && sync_ctx->dci_dm[dm_index] != NULL && sync_ctx->dci_dm_mr[dm_index] != NULL);

	assert(size < 511 && source_buf != NULL && source_offset % 512 == 0 && dest_offset % 512 == 0);

	// 用来拷贝数据到网卡
    struct ibv_exp_memcpy_dm_attr cpy_attr;
    memset(&cpy_attr, 0, sizeof(cpy_attr));
    // 初始化要拷贝的其他部分
    cpy_attr.memcpy_dir = IBV_EXP_DM_CPY_TO_DEVICE;
    cpy_attr.host_addr = source_buf + source_offset;
    cpy_attr.length = size;
    cpy_attr.dm_offset = dest_offset;

	ibv_exp_memcpy_dm(sync_ctx->dci_dm[dm_index], &cpy_attr);
}

void cpy_to_device(struct orcl_context* orcl_ctx, uint8_t* source_buf, int source_offset, int dm_index, int dest_offset, int size){
	assert(orcl_ctx != NULL && orcl_ctx->dci_dm[dm_index] != NULL && orcl_ctx->dci_dm_mr[dm_index] != NULL);

	assert(size < 511 && source_buf != NULL && source_offset % 512 == 0 && dest_offset % 512 == 0);

	// 用来拷贝数据到网卡
    struct ibv_exp_memcpy_dm_attr cpy_attr;
    memset(&cpy_attr, 0, sizeof(cpy_attr));
    // 初始化要拷贝的其他部分
    cpy_attr.memcpy_dir = IBV_EXP_DM_CPY_TO_DEVICE;
    cpy_attr.host_addr = source_buf + source_offset;
    cpy_attr.length = size;
    cpy_attr.dm_offset = dest_offset;

	ibv_exp_memcpy_dm(orcl_ctx->dci_dm[dm_index], &cpy_attr);
}